﻿using DownLoadTextByScrapy.Model;
using HtmlAgilityPack;
using ScrapySharp.Extensions;
using Serilog;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Security.Policy;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Windows.Forms;
using static System.Net.Mime.MediaTypeNames;
using static System.Net.WebRequestMethods;
using File = System.IO.File;
using HtmlDocument = HtmlAgilityPack.HtmlDocument;


namespace DownLoadTextByScrapy
{
    public partial class Form1 : Form
    {
        const string DEST_DIR_NAME = "Files";

        const string HOME_URL = "https://www.biiqu.com/";
        string _destDir = string.Empty;

        private delegate T DoActin<T>(T obj, ref bool bContinue);

        public Form1()
        {
            InitializeComponent();
            Log.Logger = new LoggerConfiguration()
                .MinimumLevel.Verbose()
                .WriteTo.Console(outputTemplate: "{Timestamp:yyyy-MM-dd HH:mm:ss.fff} [{Level:u3}] {SourceContext} - {Message:lj}{NewLine}{Exception}")
                .WriteTo.File(Path.Combine("logs", @"log.txt"), rollingInterval: RollingInterval.Day, retainedFileCountLimit: 3, outputTemplate: "{Timestamp:yyyy-MM-dd HH:mm:ss.fff} [{Level:u3}] {SourceContext} - {Message:lj}{NewLine}{Exception}")
                .CreateLogger();

            _destDir = Path.Combine(Environment.CurrentDirectory, DEST_DIR_NAME);

        }

        private List<Chapter> GetAllChapterLink(string url)
        {
            Log.Information($"url:{url}");

            List<Chapter> lstChapters = new List<Chapter>();
            string content = GetUrlContent(url);
            if (string.IsNullOrEmpty(content))
            {
                Log.Information($"content:Is Emptry");
                return lstChapters;
            }

            HtmlDocument doc = new HtmlDocument();
            doc.LoadHtml(content);
            HtmlNode htmlNode = doc.DocumentNode;
            IEnumerable<HtmlNode> titles = htmlNode.CssSelect(".listmain > dl dd > a");//html中的div class=title标题

            int index = 0;

            string subItem = ".html";
            for (int i = 0; i < titles.Count(); i++)
            {
                var item = titles.ElementAt(i);
                string path = item.Attributes["href"].Value;
                if (!path.EndsWith(subItem))
                {
                    continue;
                }

                var u = new Uri(HOME_URL + path);
                string sIndex = u.AbsolutePath.Split('/').Last();
                sIndex = sIndex.Substring(0, sIndex.Length - subItem.Length);
                if (!int.TryParse(sIndex, out index))
                {
                    continue;
                }

                Chapter c = new Chapter
                {
                    Title = item.InnerText,
                    Url = HOME_URL + path,
                    Index = index,
                };

                index++;

                Log.Information($"GetAllChapterLink---index: {i}, chapter: {c.Obj2Str()}");
                lstChapters.Add(c);
            }


            return lstChapters;
        }

        private string GetUrlContent(string url)
        {
            string content = string.Empty;
            try
            {
                HttpWebRequest request = WebRequest.Create(url) as HttpWebRequest;
                request.CookieContainer = new CookieContainer();
                request.Timeout = 10 * 1000;
                request.UserAgent = "Code Sample Web Client";
                request.Credentials = CredentialCache.DefaultCredentials;
                HttpWebResponse response = request.GetResponse() as HttpWebResponse;
                StreamReader sr = new StreamReader(response.GetResponseStream(), Encoding.UTF8);
                content = sr.ReadToEnd();
                sr.Close();
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.Message);
                Log.Warning($"GetUrlContent---url:{ url}, ex: {ex.Message}" );
            }

            return content;
        }

        private void button1_Click(object sender, EventArgs e)
        {
            if (Directory.Exists(_destDir))
            {
                Directory.Delete(_destDir, true);
            }
            Directory.CreateDirectory(_destDir);

            string url = edt_url.Text;
            List<Chapter> lstChatpers =GetAllChapterLink(url);
            if (lstChatpers.Count <= 0)
            {
                return;
            }

            Log.Information($"button1_Click---LoadContent Start");
            lstChatpers = RunTask(lstChatpers, func: new Func<Chapter, Chapter>(LoadContent), check: CheckContinue);
            Log.Information($"button1_Click---LoadContent End");

            MessageBox.Show("文件下载成功！");
        }

        private bool CheckContinue(Chapter c)
        {
            if (c == null)
            {
                Log.Warning($"CheckContinue---c == null");
                return false;
            }

            if (c.RetryCount >= 5 )
            {
                Log.Information($"CheckContinue---Chapter: [{c.Obj2Str()}] Load Error");
                return false;
            }

            if (c.RetryCount == 0)
            {
                Log.Information($"CheckContinue---Chapter: [{c.Obj2Str()}] Load Success");
                return false;
            }

            Log.Information($"CheckContinue---url({c.Url}) RetryCount({c.RetryCount}) Load Error");
            return true;
        }


        private bool LoadChapterContent(ref List<Chapter> lstChatpers)
        {
            bool bRet = false;

            //var subLst = lstChatpers.GetRange(0, 5);
            lstChatpers = RunTask(lstChatpers, func: new Func<Chapter, Chapter>(LoadContent));

            return bRet;
        }

        public static void RunTask<T>(List<T> list, Action<T> action, int threadCount = 16, bool waitFlag = true)
        {
            ConcurrentQueue<T> queue = new ConcurrentQueue<T>(list);
            Task[] tasks = new Task[threadCount];
            for (int i = 0; i < threadCount; i++)
            {
                tasks[i] = Task.Run(() =>
                {
                    T t;
                    while (queue.TryDequeue(out t))
                    {
                        action(t);
                    }
                });
            }
            if (waitFlag)
            {
                Task.WaitAll(tasks);
            }
        }

        public static List<T> RunTask<T>(List<T> list, Func<T, T> func, int threadCount = 16)
        {
            var result = new List<T>();
            ConcurrentQueue<T> queue = new ConcurrentQueue<T>(list);
            Task<List<T>>[] tasks = new Task<List<T>>[threadCount];
            for (int i = 0; i < threadCount; i++)
            {
                tasks[i] = Task.Run<List<T>>(() =>
                {
                    var rList = new List<T>();
                    while (queue.TryDequeue(out T t))
                    {
                        var obj = func(t);
                        if (obj == null)
                        {
                            queue.Enqueue(t);
                            Thread.Sleep(50);
                            continue;
                        }

                        rList.Add(obj);
                        Thread.Sleep(10);
                    }
                    return rList;
                });
            }
            Task.WaitAll(tasks);
            for (int i = 0; i < threadCount; i++)
            {
                result.AddRange(tasks[i].Result);
            }
            return result;
        }


        public static List<T> RunTask<T>(List<T> list, Func<T, T> func, Func<T, bool> check, int threadCount = 8)
        {
            var result = new List<T>();
            ConcurrentQueue<T> queue = new ConcurrentQueue<T>(list);
            Task<List<T>>[] tasks = new Task<List<T>>[threadCount];
            for (int i = 0; i < threadCount; i++)
            {
                tasks[i] = Task.Run<List<T>>(() =>
                {
                    Random r = new Random(int.Parse(DateTime.Now.ToString("mmssfff")));
                    var rList = new List<T>();
                    while (queue.TryDequeue(out T t))
                    {
                        var obj = func(t);

                        bool bContinue = check(obj);
                        if (bContinue)
                        {
                            int num = r.Next(50, 80);
                            queue.Enqueue(t);
                            Thread.Sleep(num);
                            continue;
                        }

                        rList.Add(obj);
                        int num1 = r.Next(10, 30);
                        Thread.Sleep(num1);
                    }
                    return rList;
                });
            }
            Task.WaitAll(tasks);
            for (int i = 0; i < threadCount; i++)
            {
                result.AddRange(tasks[i].Result);
            }
            return result;
        }

        private Chapter LoadContent(Chapter c)
        {
            Log.Information($"chapter: {c.Obj2Str()}");
            string content = GetUrlContent(c.Url);
            if (string.IsNullOrEmpty(content))
            {
                Log.Information($"LoadContent---content Empty");
                c.RetryCount++;
                return c;
            }

            HtmlDocument doc = new HtmlDocument();
            doc.LoadHtml(content);
            if (string.IsNullOrEmpty(doc.Text))
            {
                c.RetryCount++;
                Log.Information($"LoadContent---Doc Load Error");
                return c;
            }

            HtmlNode htmlNode = doc.DocumentNode;
            HtmlNode node = htmlNode.SelectSingleNode("//*[@id=\"chaptercontent\"]");
            List<string> delXpath = new List<string> { "//*[@class=\"readinline\"]" };
    
            foreach (var path in delXpath)
            {
                var n = node.SelectSingleNode(path);
                if (n != null)
                {
                    node.ChildNodes.Remove(n);
                }
            }

            string txt = node.InnerText;

            string file = Path.Combine(_destDir, $"{c.Index}.txt");
            using (StreamWriter sw = new StreamWriter(file, true))
            {
                string result = txt.Replace("&nbsp;&nbsp;", " ").Replace("\r\r", Environment.NewLine).Replace("请收藏本站：https://www.biiqu.com。笔趣阁手机版：https://m.biiqu.com ", "");              

                sw.WriteLine(c.Title);
                sw.WriteLine(result);
            }
            c.RetryCount = 0 ;
            return c;
        }

        private void btn_combine_Click(object sender, EventArgs e)
        {
            if (!Directory.Exists(_destDir))
            {
                MessageBox.Show("文本目录不存在！");
                return;
            }

            string[] files = Directory.GetFiles(_destDir);            
            if (files.Count() == 0)
            {
                MessageBox.Show("文本文件为空！");
                return;
            }

            List<string> lst = files.ToList();
            lst.Sort((p1, p2) =>
            {
                string name1 = Path.GetFileNameWithoutExtension(p1);
                string name2 = Path.GetFileNameWithoutExtension(p2);
                int value1 = int.Parse(name1);
                int value2 = int.Parse(name2);
                return value1 - value2;

            });


            string path = Path.Combine(Environment.CurrentDirectory, "a.txt");

            using (StreamWriter sw = new StreamWriter(path, true))
            {
                for (int i = 0; i < lst.Count; i++)
                {
                    using (StreamReader sr = new StreamReader(lst[i]))
                    {
                        sw.Write(sr.ReadToEnd());
                    }
                }
            }

            MessageBox.Show("文件合并成功！");

        }
    }

    public static class UtilsHelper
    {
        public static string Obj2Str<T>(this T obj) where T : class
        {
            string json = Newtonsoft.Json.JsonConvert.SerializeObject(obj);
            return json;
        }

    }

}
